package cn.net.withub.dataCollector.web.collector;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import cn.net.withub.dataCollector.common.editor.DateUtil;
import cn.net.withub.dataCollector.common.model.*;
import cn.net.withub.dataCollector.common.utils.CreateNewKey;
import cn.net.withub.dataCollector.common.utils.HTMLSpirit;
import cn.net.withub.dataCollector.web.service.CollectorService;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.web.context.WebApplicationContext;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

/**
 * Created by hupeng on 2018/6/8.
 */
public class CollectorUtilsM41 extends BreadthCrawler {

    //@Resource
    private CollectorService collectorService;

    //@Resource
    private WebApplicationContext webAppliction;

    private TCollectorConfig tCollectorConfig;

    private Logger log = Logger.getLogger(this.getClass());

    public void CollectorUtils(CollectorService collectorService){
        collectorService = collectorService;
    }

    public CollectorUtilsM41(String crawlPath, boolean autoParse, WebApplicationContext wac, TCollectorConfig config) {
        super(crawlPath, autoParse);

        webAppliction = wac;
        if(webAppliction !=null){
            collectorService = (CollectorService)webAppliction.getBean("collectorService");
        }
        if(config != null){
            tCollectorConfig = config;
        }
    }

    public void visit(Page page, CrawlDatums crawlDatums) {
        //Config.TIMEOUT_CONNECT = 20000;
        /*读取超时*/
        //Config.TIMEOUT_READ = 20000;
        String url = page.getUrl();
        String html = page.getHtml();
        String contentType = page.getResponse().getContentType();
        //log.info("--------------------------URL:\n" + url+"     "+contentType);
        try{
            //http://149.62.0.202/Category.asp?ID=25  http://149.62.0.202/Category.asp?ID=25&Page=2  http://149.62.0.202/Category.asp?ID=26
            if(page.matchUrl("http://.*/Category.asp\\?ID=.*") || page.matchUrl("http://.*/Category.asp\\?ID=.*&Page=.*")){
                //log.info("--------------------------html:\n"+page.html());
                //获取新闻列表内容
                Elements liElements = page.select("div[class=sortlist] ul li:has(a[href][style][target=_blank][title])");
                //log.info("liElements:"+liElements);
                for (Element li : liElements) {
                    String fbsj = "";
                    String title = "";
                    String src = "";
                    try{
                        //获取发布时间
                        Elements spanElements = li.select("span");
                        if(spanElements != null && spanElements.size()>0){
                            fbsj = spanElements.get(0).text().replaceAll("  ","").replaceAll(" ","");
                        }
                        fbsj = fbsj.replace(".","-");
                        log.info("fbsj...."+fbsj);

                        //获取发布标题
                        Elements titleElements = li.select("a[href][style][target=_blank][title]");
                        if(titleElements != null && titleElements.size()>0){
                            title = titleElements.attr("title");
                            if(null == title || "".equals(title))
                                title = titleElements.get(0).text();
                        }
                        log.info("title...."+title);

                        Elements elements = li.select("a[href][style][title][target=_blank]");
                        log.info("-----------elements:"+elements+".........");
                        for (Element element : elements) {
                            //a标签 href
                            src = element.attr("abs:href");
                            log.info("href url:\n"+src);
                            if(tCollectorConfig.getFbsj()==null || (tCollectorConfig.getFbsj()!=null && DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(tCollectorConfig.getFbsj()))>0)){
                                log.info("fbsj: "+fbsj);
                                crawlDatums.add(
                                        new CrawlDatum(src)
                                                .meta("title",title)
                                                .meta("fbsj",fbsj)
                                                .meta("firstUrl",url)
                                );
                            }
                            //判断已抓取过的最大发布日期，根据标题和发布时间进行对比，未抓取过的进行抓取
                            else if(tCollectorConfig.getFbsj()!=null && DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(tCollectorConfig.getFbsj()))==0){
                                log.info("fbsj: "+fbsj+"相同");
                                TCollectorData tCollectorData = (TCollectorData)collectorService.load(TCollectorData.class," fbsj='"+fbsj+"' and title='"+title+"'");
                                if(tCollectorData == null){
                                    log.info("fbsj: "+fbsj+"抓取");
                                    crawlDatums.add(
                                            new CrawlDatum(src)
                                                    .meta("title",title)
                                                    .meta("fbsj",fbsj)
                                                    .meta("firstUrl",url)
                                    );
                                }
                            }

                        }
                    }catch (Exception e){
                        saveLog(title,fbsj,url,src,0);
                        e.printStackTrace();
                        continue;
                    }
                }

                //获取下一页
                String src = "";
                try{
                    Elements pageElements = page.select("div[class=page] a[href]");
                    for(Element element : pageElements){
                        src = element.attr("abs:href");
                        String text = element.text().trim();
                        if(text.equals("下一页")){
                            log.info("抓取下一页......"+src);
                            crawlDatums.add(
                                    new CrawlDatum(src)
                            );
                        }
                    }
                }catch (Exception e){
                    saveLog("","",url,src,0);
                    e.printStackTrace();
                }
            } else if(page.matchUrl("http://.*/html/.*/.*.html")){
                //http://149.62.0.202/html/gzdt/fyzx/18/04/6415.html
                //http://149.62.0.202/html/szxw/18/06/6605.html  http://.*/UpDocument/.*.htm"
                log.info("contentType:     \n"+contentType+"     meta:"+page.meta("title"));
                Map<String,Object> map = new HashMap<String, Object>();
                String content = "";
                String serverFilePath = "";
                String wjgs = "";

                String title = page.meta("title");
                String fbsj = page.meta("fbsj").replaceAll("  ","").replaceAll(" ","").replace(".","-");
                String firstUrl = page.meta("firstUrl");

                int zt = 0;
                log.info("contentType:"+contentType);

                if(contentType.contains("msword") || contentType.contains("pdf")){
                    try{
                        //获取word文件
                        if(contentType.contains("msword")){
                            map = HttpClientUtils.getWord(url);
                            wjgs = "doc";
                        }
                        //获取pdf文件
                        else if(contentType.contains("pdf")){
                            map = HttpClientUtils.getPdf(url);
                            wjgs = "pdf";
                        }

                        if(map != null){
                            content = map.get("content")==null?"":map.get("content").toString();
                            serverFilePath = map.get("serverFilePath")==null?"":map.get("serverFilePath").toString();
                        }

                        Date date = new Date();
                        String id = new CreateNewKey().createId();
                        TCollectorData tCollectorData = new TCollectorData();
                        tCollectorData.setId(id);
                        tCollectorData.setConfigId(tCollectorConfig.getId());
                        tCollectorData.setTitle(title);
                        Calendar nowdate = Calendar.getInstance();
                        String year = String.valueOf(nowdate.get(Calendar.YEAR));
                        if(null != fbsj && !"".equals(fbsj)) {
                            if (Integer.parseInt(fbsj.substring(0, 4)) > Integer.parseInt(year)) {
                                fbsj = year + fbsj.substring(4);
                                //log.info("fbsj:" + fbsj);
                            }
                        }
                        tCollectorData.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                        tCollectorData.setWjgs(wjgs);
                        tCollectorData.setUrl(url);
                        tCollectorData.setFydm(tCollectorConfig.getFydm());
                        tCollectorData.setFymc(tCollectorConfig.getFymc());
                        tCollectorData.setWjlj(serverFilePath);
                        tCollectorData.setSystime(date);
                        collectorService.save(tCollectorData);

                        TCollectorContent tCollectorContent = new TCollectorContent();
                        tCollectorContent.setId(new CreateNewKey().createId());
                        tCollectorContent.setContent(content);
                        tCollectorContent.setSystime(date);
                        tCollectorContent.setDataId(id);
                        collectorService.save(tCollectorContent);

                        //保存最大发布时间
                        TCollectorConfig config = (TCollectorConfig)collectorService.load(TCollectorConfig.class," id='"+tCollectorConfig.getId()+"'");
                        if(config.getFbsj()==null || DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(config.getFbsj()))>0){
                            log.info("config........."+fbsj);
                            config.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                            collectorService.save(config);
                        }

                        zt = 1;
                    }catch(Exception e){
                        zt = 0;
                        e.printStackTrace();
                    }finally {
                        saveLog(title,fbsj,firstUrl,url,zt);
                    }
                }
                //html文件
                else if(contentType.contains("html")){
                    try{
                        html = page.select("div[id=showcontent]").html();
                        //获取发布相关信息 -- 发布人：本站 作者：研究室 发布日期：2018-6-7 来源：本站 浏览：116次 字号：
                        String ttt1 = page.select("td[class=text] h2").text();
                        log.info("ttt1:"+ttt1);
                        int index_fbsj = ttt1.indexOf("发布日期：");
                        int index_xxfb = ttt1.indexOf("发布人：");
                        int index_xxzz = ttt1.indexOf("作者：");
                        if(index_xxzz <= -1)
                            index_xxzz = ttt1.indexOf("作者:");
                        int index_djcs = ttt1.indexOf("浏览：");
                        int index_fbly = ttt1.indexOf("来源：");//来源
                        int index_zh =  ttt1.indexOf("字号：");//字号

                        if(index_fbsj > -1){
                            fbsj = ttt1.substring(index_fbsj+5,index_fbly).replaceAll(" ","").replaceAll("  ","").replace(".","-");
                        }
                        String xxzz = null;
                        if(index_xxzz > -1){
                            xxzz = ttt1.substring(index_xxzz+3,index_fbsj);
                        }
                        String xxfb = null;
                        if(index_xxfb > -1)
                            xxfb = ttt1.substring(index_xxfb+4,index_xxzz);
                        String djcs = null;
                        if(index_djcs > -1)
                            djcs = ttt1.substring(index_djcs+3,index_zh);
                        String fbly = null;
                        if(index_fbly > -1)
                            fbly = ttt1.substring(index_fbly+3,index_djcs);
                        Elements h3Elements = page.select("td[class=text] h2 script[src]");
                        String djcsSrc = h3Elements.get(0).attr("abs:src");
                        log.info("djcsSrc:"+djcsSrc);
                        String djcs2 = getHtml(djcsSrc);
                        if(null != djcs2 && !"".equals(djcs2)){
                            djcs = djcs2 + "次";
                        }else {
                            djcs = null;
                        }

                        //保存爬取信息
                        String tqUrl = url;

                        try{
                            Date date = new Date();
                            String id = new CreateNewKey().createId();
                            TCollectorData tCollectorData = new TCollectorData();
                            tCollectorData.setId(id);
                            tCollectorData.setConfigId(tCollectorConfig.getId());
                            tCollectorData.setTitle(title);
                            Calendar nowdate = Calendar.getInstance();
                            String year = String.valueOf(nowdate.get(Calendar.YEAR));
                            if(null != fbsj && !"".equals(fbsj)) {
                                if (Integer.parseInt(fbsj.substring(0, 4)) > Integer.parseInt(year)) {
                                    fbsj = year + fbsj.substring(4);
                                    //log.info("fbsj:" + fbsj);
                                }
                            }
                            tCollectorData.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                            tCollectorData.setXxly(fbly);
                            tCollectorData.setXxfb(xxfb);
                            tCollectorData.setXxzz(xxzz);
                            //tCollectorData.setBjsh(bjsh);
                            tCollectorData.setDjcs(djcs);
                            tCollectorData.setWjgs("html");
                            tCollectorData.setUrl(tqUrl);
                            tCollectorData.setFydm(tCollectorConfig.getFydm());
                            tCollectorData.setFymc(tCollectorConfig.getFymc());
                            tCollectorData.setSystime(date);
                            collectorService.save(tCollectorData);

                            TCollectorContent tCollectorContent = new TCollectorContent();
                            tCollectorContent.setId(new CreateNewKey().createId());
                            tCollectorContent.setContent(html);
                            tCollectorContent.setText(HTMLSpirit.getTextFromTHML(html));
                            tCollectorContent.setSystime(date);
                            tCollectorContent.setDataId(id);
                            collectorService.save(tCollectorContent);

                            //遍历所有html中所有图片
                            Elements imgs = page.select("img[src]");
                            if(null != imgs && imgs.size() > 0){
                                for (Element img : imgs) {
                                    String src = img.attr("abs:src").replace("../","");
                                    log.info("img url:\n"+src);
                                    crawlDatums.add(new CrawlDatum(src)
                                            .meta("dataId",id)
                                    );
                                }

                            }

                            //保存最大发布时间
                            TCollectorConfig config = (TCollectorConfig)collectorService.load(TCollectorConfig.class," id='"+tCollectorConfig.getId()+"'");
                            if(config.getFbsj()==null || DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(config.getFbsj()))>0){
                                config.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                                collectorService.save(config);
                            }

                            zt = 1;
                        }catch(Exception e){
                            zt = 0;
                            e.printStackTrace();
                        }finally {
                            saveLog(title,fbsj,firstUrl,tqUrl,zt);
                        }
                    }catch (Exception e){
                        saveLog(title,fbsj,firstUrl,url,0);
                        e.printStackTrace();
                    }
                }
            }else if(contentType !=null && contentType.startsWith("image")){
                String dataId = page.meta("dataId")==null?"":page.meta("dataId");
                String imageName = url.substring(url.lastIndexOf("/")+1,url.length());
                //log.info("imageName:\n"+imageName);
                String serverFilePath = "";

                try {
                    serverFilePath = HttpClientUtils.getImage(url);
                } catch (IOException e) {
                    serverFilePath = "";
                    e.printStackTrace();
                }finally {
                    TCollectorElement tCollectorElement = new TCollectorElement();
                    tCollectorElement.setId(new CreateNewKey().createId());
                    tCollectorElement.setDataId(dataId);
                    tCollectorElement.setTitle(imageName);
                    tCollectorElement.setUrl(url);
                    tCollectorElement.setWjlj(serverFilePath);
                    tCollectorElement.setSystime(new Date());
                    collectorService.save(tCollectorElement);
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }
    public  String getHtml(String httpPath){
        try {
            //建立连接
            URL url = new URL(httpPath);
            HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
            httpUrlConn.setDoInput(true);
            httpUrlConn.setRequestMethod("GET");
            httpUrlConn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
            //获取输入流
            InputStream input = httpUrlConn.getInputStream();
            //将字节输入流转换为字符输入流
            InputStreamReader read = new InputStreamReader(input, "utf-8");
            //为字符输入流添加缓冲
            BufferedReader br = new BufferedReader(read);
            // 读取返回结果
            String data = br.readLine();
            if (null != data) {
                data = data.replaceAll("[a-zA-Z]","").replaceAll("[\\pP\\pZ\\pS\\pM\\pC]","");
            }
            // 释放资源
            br.close();
            read.close();
            input.close();
            httpUrlConn.disconnect();
            //document.write('338');
            System.out.println("data:"+data);
            return  data;
        } catch (MalformedURLException e) {
            e.printStackTrace();
            return  null;
        } catch (IOException e) {
            e.printStackTrace();
            return  null;
        }
    }

    public void saveLog(String title,String fbsj,String firstUrl,String tqUrl,int zt){
        try {
            log.info("savelog........."+fbsj);
            TCollectorLog tCollectorLog = new TCollectorLog();
            tCollectorLog.setId(new CreateNewKey().createId());
            tCollectorLog.setConfigId(tCollectorConfig.getId());
            tCollectorLog.setIp(tCollectorConfig.getIp());
            tCollectorLog.setModuleName(tCollectorConfig.getModuleName());
            tCollectorLog.setFydm(tCollectorConfig.getFydm());
            tCollectorLog.setFymc(tCollectorConfig.getFymc());
            tCollectorLog.setTitle(title);
            try {
                tCollectorLog.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
            }catch (Exception e) {
                e.printStackTrace();
            }
            tCollectorLog.setZt(zt);
            tCollectorLog.setFirstUrl(firstUrl);
            tCollectorLog.setTqUrl(tqUrl);
            tCollectorLog.setSysTime(new Date());

            collectorService.save(tCollectorLog);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) throws Exception {
        //getHtml("http://149.72.0.202/www/js/gethits.asp?id=3969");

    //CollectorUtilsM1F crawler = new CollectorUtilsM1F("crawl", true,null,null);
        //crawler.setThreads(10);
        //crawler.setTopN(100);
       //crawler.setResumable(true);
        //crawler.start(500);

        /*Executor executor=new Executor() {
            public void execute(CrawlDatum datum, CrawlDatums next) throws Exception {
                HtmlUnitDriver driver = new HtmlUnitDriver();
                driver.setJavascriptEnabled(true);
                driver.get(datum.getUrl());
                //WebElement element=driver.findElementByCssSelector("ul#ssxw");
                WebElement element=driver.findElementByCssSelector("span#outlink1");
                log.info("反链数:"+element.getText());
            }
        };

        //创建一个基于伯克利DB的DBManager
        DBManager manager=new BerkeleyDBManager("crawl");
        //创建一个Crawler需要有DBManager和Executor
        Crawler crawler= new Crawler(manager,executor);
        //crawler.addSeed("http://149.0.160.10");
        crawler.addSeed("http://seo.chinaz.com/?host=www.tuicool.com");
        crawler.start(1);*/
    }

}
